{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vanilla LSTM in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we use an LSTM to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n",
    "from keras.layers import LSTM # new! \n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/vanillaLSTM'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64 \n",
    "n_unique_words = 10000 \n",
    "max_review_length = 100 # lowered due to vanishing gradient over time\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# LSTM layer architecture:\n",
    "n_lstm = 256 \n",
    "drop_lstm = 0.2\n",
    "\n",
    "# dense layer architecture: \n",
    "# n_dense = 256\n",
    "# dropout = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
    "model.add(SpatialDropout1D(drop_embed))\n",
    "model.add(LSTM(n_lstm, dropout=drop_lstm))\n",
    "# model.add(Dense(n_dense, activation='relu')) # typically don't see top dense layer in NLP like in \n",
    "# model.add(Dropout(dropout))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 100, 64)           640000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 100, 64)           0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 256)               328704    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 968,961\n",
      "Trainable params: 968,961\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 36s - loss: 0.5699 - acc: 0.6843 - val_loss: 0.3804 - val_acc: 0.8371\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 35s - loss: 0.3074 - acc: 0.8748 - val_loss: 0.3394 - val_acc: 0.8516\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 36s - loss: 0.2449 - acc: 0.9032 - val_loss: 0.3676 - val_acc: 0.8376\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 34s - loss: 0.2015 - acc: 0.9237 - val_loss: 0.4154 - val_acc: 0.8420\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f24c73a9fd0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# go have a gander at nvidia-smi\n",
    "# 85.2% validation accuracy in epoch 2\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 27s    \n"
     ]
    }
   ],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFR9JREFUeJzt3X+QXeV93/H3x8jYcWJbAhaGkUSFx4pr7BljugO4nkkd\nyxUCexB/QEaepsiMpuqkJE3STFtoO6MWTAf3FwkzMaka1AhPYiHTuGhsGqoKGLedghGGEANhtAaC\ntqJoYwm5KWMSkW//uI/wIna1Z6Xdu6zP+zWzc8/5nuec8zys2M89P+49qSokSf3zjoXugCRpYRgA\nktRTBoAk9ZQBIEk9ZQBIUk8ZAJLUUwaAJPWUASBJPWUASFJPLVnoDpzIWWedVatWrVrobkhv9YNn\nB6/v+9DC9kOawmOPPfanVTUyU7u3dQCsWrWKvXv3LnQ3pLf6b58avH7moYXshTSlJH/SpZ2ngCSp\npwwASeopA0CSesoAkKSeMgAkqac6BUCSX03yVJLvJvlqkncnOT/JI0n2Jbk7yemt7bva/FhbvmrS\ndm5s9WeTXDY/Q5IkdTFjACRZDvx9YLSqPgqcBmwAvgTcVlWrgcPAprbKJuBwVX0QuK21I8kFbb2P\nAOuALyc5bW6HI0nqquspoCXATyRZArwHeAn4NHBPW74duKpNr2/ztOVrkqTVd1TVa1X1PDAGXHzq\nQ5AknYwZA6Cq/jfwb4AXGfzhPwI8BrxSVUdbs3FgeZteDuxv6x5t7c+cXJ9inTck2Zxkb5K9ExMT\nJzMmSVIHM34SOMkyBu/ezwdeAb4GXD5F02NPl880y6arv7lQtRXYCjA6OuoT6yUtmFU3fHPB9v3C\nrZ+d9310OQX0GeD5qpqoqr8Afh/468DSdkoIYAVwoE2PAysB2vL3A4cm16dYR5I0ZF0C4EXg0iTv\naefy1wBPAw8CV7c2G4F72/SuNk9b/kBVVatvaHcJnQ+sBr49N8OQJM3WjKeAquqRJPcA3wGOAo8z\nOEXzTWBHki+22p1tlTuBryQZY/DOf0PbzlNJdjIIj6PA9VX1+hyPR5LUUadvA62qLcCW48rPMcVd\nPFX1Q+CaabZzC3DLLPsoSZoHfhJYknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeop\nA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmnOj0PYLFaqOd5DuNZnpJ0qjwCkKSemjEA\nknwoyROTfn6Q5FeSnJFkd5J97XVZa58ktycZS/JkkosmbWtja78vycbp9ypJmm8zBkBVPVtVF1bV\nhcBfA14Fvg7cAOypqtXAnjYPcDmDB76vBjYDdwAkOYPBYyUvYfAoyS3HQkOSNHyzPQW0BvheVf0J\nsB7Y3urbgava9Hrgrhp4GFia5FzgMmB3VR2qqsPAbmDdKY9AknRSZhsAG4CvtulzquolgPZ6dqsv\nB/ZPWme81aarv0mSzUn2Jtk7MTExy+5JkrrqHABJTgeuBL42U9MpanWC+psLVVurarSqRkdGRrp2\nT5I0S7M5Argc+E5VvdzmX26ndmivB1t9HFg5ab0VwIET1CVJC2A2AfB5fnT6B2AXcOxOno3AvZPq\n17a7gS4FjrRTRPcDa5Msaxd/17aaJGkBdPogWJL3AH8T+LuTyrcCO5NsAl4Ermn1+4ArgDEGdwxd\nB1BVh5LcDDza2t1UVYdOeQSSpJPSKQCq6lXgzONq32dwV9DxbQu4fprtbAO2zb6bkqS55ieBJamn\nDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmn\nDABJ6ikDQJJ6ygCQpJ4yACSppzoFQJKlSe5J8sdJnknyiSRnJNmdZF97XdbaJsntScaSPJnkoknb\n2dja70uycfo9SpLmW9cjgN8A/qCq/irwMeAZ4AZgT1WtBva0eYDLgdXtZzNwB0CSM4AtwCXAxcCW\nY6EhSRq+GQMgyfuAnwHuBKiqP6+qV4D1wPbWbDtwVZteD9xVAw8DS5OcC1wG7K6qQ1V1GNgNrJvT\n0UiSOutyBPABYAL4j0keT/LbSX4SOKeqXgJor2e39suB/ZPWH2+16epvkmRzkr1J9k5MTMx6QJKk\nbroEwBLgIuCOqvo48P/40emeqWSKWp2g/uZC1daqGq2q0ZGRkQ7dkySdjC4BMA6MV9Ujbf4eBoHw\ncju1Q3s9OKn9yknrrwAOnKAuSVoAMwZAVf0fYH+SD7XSGuBpYBdw7E6ejcC9bXoXcG27G+hS4Eg7\nRXQ/sDbJsnbxd22rSZIWwJKO7X4J+N0kpwPPAdcxCI+dSTYBLwLXtLb3AVcAY8CrrS1VdSjJzcCj\nrd1NVXVoTkYhSZq1TgFQVU8Ao1MsWjNF2wKun2Y724Bts+mgJGl++ElgSeopA0CSesoAkKSeMgAk\nqacMAEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAk\nqac6BUCSF5L8UZInkuxttTOS7E6yr70ua/UkuT3JWJInk1w0aTsbW/t9STZOtz9J0vybzRHAz1bV\nhVV17MlgNwB7qmo1sKfNA1wOrG4/m4E7YBAYwBbgEuBiYMux0JAkDd+pnAJaD2xv09uBqybV76qB\nh4GlSc4FLgN2V9WhqjoM7AbWncL+JUmnoGsAFPBfkzyWZHOrnVNVLwG017NbfTmwf9K64602XV2S\ntAA6PRQe+GRVHUhyNrA7yR+foG2mqNUJ6m9eeRAwmwHOO++8jt2TJM1WpyOAqjrQXg8CX2dwDv/l\ndmqH9nqwNR8HVk5afQVw4AT14/e1tapGq2p0ZGRkdqORJHU2YwAk+ckk7z02DawFvgvsAo7dybMR\nuLdN7wKubXcDXQocaaeI7gfWJlnWLv6ubTVJ0gLocgroHODrSY61/72q+oMkjwI7k2wCXgSuae3v\nA64AxoBXgesAqupQkpuBR1u7m6rq0JyNRJI0KzMGQFU9B3xsivr3gTVT1Au4fpptbQO2zb6bkqS5\n5ieBJamnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknrKAJCk\nnjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppzoHQJLTkjye5Btt/vwkjyTZl+TuJKe3+rva/FhbvmrS\nNm5s9WeTXDbXg5EkdTebI4BfBp6ZNP8l4LaqWg0cBja1+ibgcFV9ELittSPJBcAG4CPAOuDLSU47\nte5Lkk5WpwBIsgL4LPDbbT7Ap4F7WpPtwFVten2bpy1f09qvB3ZU1WtV9TyDh8ZfPBeDkCTNXtcj\ngF8H/hHwl23+TOCVqjra5seB5W16ObAfoC0/0tq/UZ9iHUnSkM0YAEk+Bxysqscml6doWjMsO9E6\nk/e3OcneJHsnJiZm6p4k6SR1OQL4JHBlkheAHQxO/fw6sDTJktZmBXCgTY8DKwHa8vcDhybXp1jn\nDVW1tapGq2p0ZGRk1gOSJHUzYwBU1Y1VtaKqVjG4iPtAVf0t4EHg6tZsI3Bvm97V5mnLH6iqavUN\n7S6h84HVwLfnbCSSpFlZMnOTaf1jYEeSLwKPA3e2+p3AV5KMMXjnvwGgqp5KshN4GjgKXF9Vr5/C\n/iVJp2BWAVBVDwEPtennmOIunqr6IXDNNOvfAtwy205KkuaenwSWpJ4yACSppwwASeopA0CSesoA\nkKSeMgAkqacMAEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoA\nkKSemjEAkrw7ybeT/GGSp5L8i1Y/P8kjSfYluTvJ6a3+rjY/1pavmrStG1v92SSXzdegJEkz63IE\n8Brw6ar6GHAhsC7JpcCXgNuqajVwGNjU2m8CDlfVB4HbWjuSXMDg+cAfAdYBX05y2lwORpLU3YwB\nUAN/1mbf2X4K+DRwT6tvB65q0+vbPG35miRp9R1V9VpVPQ+MMcUzhSVJw9HpGkCS05I8ARwEdgPf\nA16pqqOtyTiwvE0vB/YDtOVHgDMn16dYR5I0ZJ0CoKper6oLgRUM3rV/eKpm7TXTLJuu/iZJNifZ\nm2TvxMREl+5Jkk7CrO4CqqpXgIeAS4GlSZa0RSuAA216HFgJ0Ja/Hzg0uT7FOpP3sbWqRqtqdGRk\nZDbdkyTNQpe7gEaSLG3TPwF8BngGeBC4ujXbCNzbpne1edryB6qqWn1Du0vofGA18O25GogkaXaW\nzNyEc4Ht7Y6ddwA7q+obSZ4GdiT5IvA4cGdrfyfwlSRjDN75bwCoqqeS7ASeBo4C11fV63M7HElS\nVzMGQFU9CXx8ivpzTHEXT1X9ELhmmm3dAtwy+25KkuaanwSWpJ4yACSppwwASeopA0CSesoAkKSe\nMgAkqacMAEnqKQNAknrKAJCknjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSe\n6vJM4JVJHkzyTJKnkvxyq5+RZHeSfe11Wasnye1JxpI8meSiSdva2NrvS7Jxun1KkuZflyOAo8Cv\nVdWHgUuB65NcANwA7Kmq1cCeNg9wOYMHvq8GNgN3wCAwgC3AJQweJbnlWGhIkoZvxgCoqpeq6jtt\n+v8CzwDLgfXA9tZsO3BVm14P3FUDDwNLk5wLXAbsrqpDVXUY2A2sm9PRSJI6m9U1gCSrGDwg/hHg\nnKp6CQYhAZzdmi0H9k9abbzVpqtLkhbAkq4Nk/wU8J+AX6mqHySZtukUtTpB/fj9bGZw6ojzzjuv\na/feVlbd8M0F2e8Lt352QfYrzbeF+n/qx12nI4Ak72Twx/93q+r3W/nldmqH9nqw1ceBlZNWXwEc\nOEH9Tapqa1WNVtXoyMjIbMYiSZqFLncBBbgTeKaq/t2kRbuAY3fybATunVS/tt0NdClwpJ0iuh9Y\nm2RZu/i7ttUkSQugyymgTwJ/G/ijJE+02j8BbgV2JtkEvAhc05bdB1wBjAGvAtcBVNWhJDcDj7Z2\nN1XVoTkZhSRp1mYMgKr6H0x9/h5gzRTtC7h+mm1tA7bNpoOSpPnhJ4ElqacMAEnqKQNAknrKAJCk\nnjIAJKmnDABJ6ikDQJJ6ygCQpJ4yACSppwwASeopA0CSesoAkKSeMgAkqacMAEnqKQNAknqq8zOB\n9fa3kM9N9XnE0uLjEYAk9dSMRwBJtgGfAw5W1Udb7QzgbmAV8ALwc1V1uD0/+DcYPBLyVeALVfWd\nts5G4J+1zX6xqrbP7VAkzbeFPMrU3OtyBPA7wLrjajcAe6pqNbCnzQNcDqxuP5uBO+CNwNgCXAJc\nDGxpD4aXJC2QGQOgqr4FHP/w9vXAsXfw24GrJtXvqoGHgaVJzgUuA3ZX1aGqOgzs5q2hIkkaopO9\nBnBOVb0E0F7PbvXlwP5J7cZbbbq6JGmBzPVF4ExRqxPU37qBZHOSvUn2TkxMzGnnJEk/crIB8HI7\ntUN7Pdjq48DKSe1WAAdOUH+LqtpaVaNVNToyMnKS3ZMkzeRkPwewC9gI3Npe751U/8UkOxhc8D1S\nVS8luR/4l5Mu/K4Fbjz5bkv95Z04mitdbgP9KvAp4Kwk4wzu5rkV2JlkE/AicE1rfh+DW0DHGNwG\neh1AVR1KcjPwaGt3U1Udf2FZi9hC/VHyA2jSyZsxAKrq89MsWjNF2wKun2Y724Bts+qdNIOFCp4d\nH/g+ABt8N65FzE8CS1JPGQCS1FMGgCT1lAEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBIUk8ZAJLU\nUwaAJPWUASBJPWUASFJPGQCS1FMGgCT1lAEgST1lAEhSTw09AJKsS/JskrEkNwx7/5KkgaEGQJLT\ngN8ELgcuAD6f5IJh9kGSNDDsI4CLgbGqeq6q/hzYAawfch8kSQw/AJYD+yfNj7eaJGnIlgx5f5mi\nVm9qkGwGNrfZP0vy7Enu6yzgT09y3cXKMQ/JJ96Y+tywdw3+nnshXzqlMf+VLo2GHQDjwMpJ8yuA\nA5MbVNVWYOup7ijJ3qoaPdXtLCaOuR8ccz8MY8zDPgX0KLA6yflJTgc2ALuG3AdJEkM+Aqiqo0l+\nEbgfOA3YVlVPDbMPkqSBYZ8CoqruA+4bwq5O+TTSIuSY+8Ex98O8jzlVNXMrSdKPHb8KQpJ6atEH\nwExfLZHkXUnubssfSbJq+L2cWx3G/A+SPJ3kySR7knS6JeztrOtXiCS5OkklWfR3jHQZc5Kfa7/r\np5L83rD7ONc6/Ns+L8mDSR5v/76vWIh+zpUk25IcTPLdaZYnye3tv8eTSS6a0w5U1aL9YXAh+XvA\nB4DTgT8ELjiuzd8DfqtNbwDuXuh+D2HMPwu8p03/Qh/G3Nq9F/gW8DAwutD9HsLveTXwOLCszZ+9\n0P0ewpi3Ar/Qpi8AXljofp/imH8GuAj47jTLrwD+C4PPUF0KPDKX+1/sRwBdvlpiPbC9Td8DrEky\n1QfSFosZx1xVD1bVq232YQaft1jMun6FyM3AvwJ+OMzOzZMuY/47wG9W1WGAqjo45D7OtS5jLuB9\nbfr9HPc5osWmqr4FHDpBk/XAXTXwMLA0yblztf/FHgBdvlrijTZVdRQ4Apw5lN7Nj9l+ncYmBu8g\nFrMZx5zk48DKqvrGMDs2j7r8nn8a+Okk/zPJw0nWDa1386PLmP858PNJxhncTfhLw+nagpnXr88Z\n+m2gc2zGr5bo2GYx6TyeJD8PjAJ/Y157NP9OOOYk7wBuA74wrA4NQZff8xIGp4E+xeAo778n+WhV\nvTLPfZsvXcb8eeB3qurfJvkE8JU25r+c/+4tiHn9+7XYjwBm/GqJyW2SLGFw2HiiQ663uy5jJsln\ngH8KXFlVrw2pb/NlpjG/F/go8FCSFxicK921yC8Ed/23fW9V/UVVPQ88yyAQFqsuY94E7ASoqv8F\nvJvB9wT9uOr0//vJWuwB0OWrJXYBG9v01cAD1a6uLFIzjrmdDvn3DP74L/bzwjDDmKvqSFWdVVWr\nqmoVg+seV1bV3oXp7pzo8m/7PzO44E+SsxicEnpuqL2cW13G/CKwBiDJhxkEwMRQezlcu4Br291A\nlwJHquqludr4oj4FVNN8tUSSm4C9VbULuJPBYeIYg3f+Gxaux6eu45j/NfBTwNfa9e4Xq+rKBev0\nKeo45h8rHcd8P7A2ydPA68A/rKrvL1yvT03HMf8a8B+S/CqDUyFfWMxv6JJ8lcEpvLPadY0twDsB\nquq3GFznuAIYA14FrpvT/S/i/3aSpFOw2E8BSZJOkgEgST1lAEhSTxkAktRTBoAk9ZQBIEk9ZQBI\nUk8ZAJLUU/8fAPhY5EeGsl0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f24c5979710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'93.13'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
